!pip install -q openai==1.61.1

# @title Loading the `config.json` file
import json, os

# Load the JSON file and extract values
file_name = 'config.json'
with open(file_name, 'r') as file:
    config = json.load(file)
    os.environ['OPENAI_API_KEY']  = config['API_KEY'] # Loading the API Key
    os.environ["OPENAI_BASE_URL"] = config['OPENAI_API_BASE'] # Loading the API Base Url

model_name = "gpt-4o-mini"

from openai import OpenAI

# Initialize OpenAI client
client = OpenAI()

# @title LLM function
# @markdown Once the API details are filled, the notebook will automatically load the configuration, and learners can generate model outputs using the llm() function.


def llm(system_prompt, user_prompt):
  try:
      # Craft the messages to pass to chat.completions.create
      prompt = [
          {'role':'system', 'content': system_prompt},
          {'role': 'user', 'content': user_prompt}
      ]

      response = client.chat.completions.create(
          model=model_name,
          messages=prompt,
          temperature=0
      )

      return response.choices[0].message.content.strip()

  except Exception as e:
      prediction = f'Sorry, I encountered the following error: \n {e}'
      print(prediction)

# @title Step 1: Load the Dataset
# Data Loading

import pandas as pd
df = pd.read_csv("Alex_emails_march_04.csv", index_col="email_id", encoding='latin-1')      #Add the data file location
df.head(5)

# @title Step 2: Apply Yesterbox Filtering
# @markdown The Yesterbox approach involves processing emails from the previous day first before tackling today's emails.

# @markdown For this dataset, consider today's date as 4th March 2025.

# @markdown We filter the dataset to only include emails received on 3rd March 2025 (yesterday)
# (Yesterbox Approach)(Today: 4 march)



from datetime import datetime, timedelta

yesterday_date = pd.to_datetime("3/3/2025").strftime('%m/%d/%Y')

df['date_received'] = pd.to_datetime(df['date_received']).dt.strftime('%m/%d/%Y')

yesterday_emails = df[df['date_received'] == yesterday_date].reset_index(drop=True)
print(f"Filtered Emails Count: {len(yesterday_emails)}")

Filtered Emails Count: 51

# Here we see only 51 emails, as in 10 email had the date of 4th March 2025
yesterday_emails.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 51 entries, 0 to 50
Data columns (total 5 columns):
 #   Column          Non-Null Count  Dtype 
---  ------          --------------  ----- 
 0   date_received   51 non-null     object
 1   sender          51 non-null     object
 2   subject         51 non-null     object
 3   body            51 non-null     object
 4   main_recipient  51 non-null     object
dtypes: object(5)
memory usage: 2.1+ KB

# @title Code to add categories to the dataset
categories = ['Urgent & High-Priority Emails',
              'Deadline-Driven Emails',
              'Routine Updates & Check-ins',
              'Non-Urgent Informational Emails',
              'Personal & Social Emails',
              'Spam/Unimportant Emails']

system_prompt = f"""

you are an email assistant. you will be given a dataset of emails.
your task is to categorize the emails in to the following categories:
Urgent & High-Priority Emails, Deadline-Driven Emails, Routine Updates & Check-ins, Non-Urgent Informational Emails, Personal & Social Emails, Spam/Unimportant Emails

Email :  Rolex Watch for Sale on 50% Discount
Response : 'Spam/Unimportant Emails'

Email : Project Deadline Tomorrow: Final Review Needed
Response : Deadline-Driven Emails

Email : 'Weekly Team Meeting: Agenda and Updates
Response : 'Routine Updates & Check-ins

Email : Important: Security Breach Detected in Your Account
Response : Urgent & High-Priority Emails

Email : Lunch Plans?
Response : Personal & Social Emails

Email: Your Subscription is Ending Soon in 20 days
Response : Non-Urgent Informational Emails

All of these catgorises are mutually exclusive, meaning that an email can only belong to one category.
these categories may or may not

"""

user_prompt = f"""

classify the following email. Return only the category.
use the subject and body of the email to classify it.

"""

row = yesterday_emails.iloc[0]
index = row.name  # Optional: get the index of that row

prompt = f"""{user_prompt}
Please find the attached email below from yesterday that needs to be analyzed:

{row.to_string()}
"""

# @title Categorizing the emails
from tqdm import tqdm  # Import the tqdm library for the progress bar

# Ensure the 'category' column exists in the DataFrame
if 'category' not in yesterday_emails.columns:
  yesterday_emails['category'] = None  # Create the column if it does not exist


for index, row in tqdm(yesterday_emails.iterrows(), total=yesterday_emails.shape[0], desc='Processing emails'):
  prompt = f"""{user_prompt}
    Please find the attached email below from yesterday that need to be analyzed:
    ```
    {row.to_string()}
    ```
  """

  category_by_llm = llm(system_prompt, prompt)

  # Append the category generated by the LLM to the 'category' column in the same row
  if category_by_llm in categories:
    yesterday_emails.at[index, 'category'] = category_by_llm
  else:
    yesterday_emails.at[index, 'category'] = ""

Processing emails: 100%|██████████| 51/51 [01:16<00:00,  1.50s/it]

yesterday_emails.category.value_counts()

category
Routine Updates & Check-ins        20
Urgent & High-Priority Emails      14
Deadline-Driven Emails              7
Non-Urgent Informational Emails     6
Spam/Unimportant Emails             4
Name: count, dtype: int64

system_prompt = f"""

You are the AI Email Secretary.
You will receive a DataFrame as a string. The `category` column contains the following values:

- 'Urgent & High-Priority Emails'
- 'Deadline-Driven Emails'
- 'Routine Updates & Check-ins'
- 'Non-Urgent Informational Emails'
- 'Personal & Social Emails'
- 'Spam/Unimportant Emails'

Your task is to generate an Executive Summary of Emails in the exact format shown below.
Make sure all the headings and bullet point heading are in bold.

---

Executive Summary of Emails

1. Total Number of Emails Received: <>
2. Total Number of Emails from Yesterday: <> (all emails are dated <yesterday_date>)
3. Email Breakdown by Categories (Count Only):
    Urgent & High-Priority Emails: <>
    Deadline-Driven Emails: <>
    Routine Updates & Check-ins: <>
    Non-Urgent Informational Emails: <>
    Personal & Social Emails: <>
    Spam/Unimportant Emails: <>
4. AI Conclusion:
    Critical Emails Requiring Immediate Attention:
      Urgent & High-Priority Emails: <>
      Deadline-Driven Emails: <>
      Total Critical Emails: <>
    Emails That Can Be Reviewed Later:
      Routine Updates & Check-ins: <>
      Non-Urgent Informational Emails: <>
      Personal & Social Emails: <>
      Spam/Unimportant Emails: <>
      Total Non-Critical Emails: <>

Summary Insights:
<add 2/3 lines of highlights mentioning the count of Critical and Non-Critical emails count if they are non zero. Also suggest user to take care of critical emails first and their impact.>
---
## rules
1. Fill the <> with the respective count in the data.
2. Use `category` column for calculations.
2. Make sure the count is correct.
3. Sum of count of categories should match total number of emails.

"""

# @title User Prompt
# Write your user prompt here
user_prompt = f"""
Analyze the data, count emails by `category` column.
Please generate the executive summary for emails. 
today's date is 4th March 2025, and yesterday date is 3rd March 2025.

```
{yesterday_emails.groupby(['date_received', 'category']).size().reset_index(name='count').to_string()}
```

"""

# @title  Calling the model and display the summary


response_1 = llm(system_prompt, user_prompt)        # llm is the model using gpt-4o-mini
response_1

from IPython.display import display, Markdown
display(Markdown(response_1))

# @title System Prompt

# Write your system prompt here
system_prompt = """

You are an AI Email assitant, when a email data is provided to you,
you will make the summary of the each email individually,
format it like below. <> contains the instruction on how to write it.

Begain the output by mentioning about category mentioned by user add a 1/2 line intro on top before the output starts.
Also add nature of emails and importance.

Then mentioed the category for which data is given.

Summarize each and every email provided in the data.

For each email, use numbered bullets starting from 1.
Include below fields with each numbered bullet starting from 1, in the below order and format.

Make sure all the headings and bullet point heading are in bold including numbered bullet.

Subject: <subject of the email proccesssing> \n
Received: <date of recieving> \n
Sender Name: <reciever's name> \n
Summary: <summary of email. what was mentioed in the email> \n
Next Step: <based on understanding from subject and summary what next steps required from reciever>

Keep the format as is and serial number in order. also mention the category given in the data or user prompt on the top of the output.

Example output:

Here are the Urgent & High-Priority emails that require your attention or intervention, as per the provided records. All emails listed were received yesterday and are part of today's to-do list.

Urgent & High-Priority emails

1. Subject: URGENT: Medication Alerts Not Firing - This is Dangerous
   Received: 03/03/2025
   Sender Name: Rachel Thompson
   Summary: Rachel highlights a serious issue where automated medication alerts are not firing for ICU patients, posing a safety risk.
   Next Step: Join the emergency call and prioritize fixing the issue immediately.

"""

# Filtering out the emails that are urgent and high-priority
cat_to_be_summarised = 'Urgent & High-Priority Emails'
urgent_emails = yesterday_emails[yesterday_emails['category'] == cat_to_be_summarised]

# @title User Prompt

# Write your user prompt here
user_prompt = f"""


As email assistant, summarize the emails for {cat_to_be_summarised}. Make sure all the emails are dated {yesterday_date} and in today's to do list.

Below is the attached DataFrame, which contains all the emails that needs to be summarized:
```
{urgent_emails.to_string()}

```
"""

# @title  Calling the model and display the summary

response_2 = llm(system_prompt, user_prompt)

from IPython.display import display, Markdown
display(Markdown(response_2))

# @title System Prompt

# Write your system prompt here
system_prompt = """

You are an AI Email assitant, when email data is provided to you,
you will make the summary of the each email individually,
format it like below. <> contains the instruction on how to write it.

Make sure all the headings and bullet points are in bold.
Each eamail which have one of the headings Email Exceeding Deadline or Email Requiring Action Today.
Use the following output structure for each email. Make sure NO email is left out in the data.

Deadline-Driven Emails Summary

1. Email Exceeding Deadline
   - Subject: <subject of the email processing>
   - Received: <date of receiving>
   - Sender Name: <receiver's  name>
   - Summary: <summary of email. what was mentioed in the email> 
   - Next Step: <based on understanding from subject and summary what next steps required from receiver>

2  Email Requiring Action Today
   - Subject: <subject of the email processing> 
   - Received: <date of receiving> 
   - Sender Name: <receiver's name> 
   - Summary: <summary of email. what was mentioed in the email> 
   - Next Step: <based on understanding from subject and summary what next steps required from receiver>



Final Count of Deadline-Driven Emails
   - Total Deadline-Driven Emails: total number of emails also mention in brackets if there are emails exceeding deadline.

Summary of Actionable Items
   <bullet pointed summary of actionable item today>

### Definition of fields given in output
- Email Exceeding Deadline --> emails whose expected action or response deadline has already passed.
- Email Requiring Action Today --> rest of the emails which are dealine driven and require action today.

### Rules
- Summarize each and every email provided in the data. Make sure no email is left from the data.
- Each eamail which have one of the heading Email Exceeding Deadline or Email Requiring Action Today in bold letter.
- Structure should be exactly as given above. 
- Put Email Exceeding Deadline first and then Email Requiring Action Today.
- Make sure you are comparing the email respond action date and yesterday date. 
- Put email under Email Exceeding Deadline only if, action date is scrictly before today's date.

### Example output for email summary and next step output: 

Subject: URGENT: Medication Alerts Not Firing - This is Dangerous
Received: 03/03/2025
Sender Name: Rachel Thompson
Summary: Rachel highlights a serious issue where automated medication alerts are not firing for ICU patients, posing a safety risk.
Next Step: Join the emergency call and prioritize fixing the issue immediately.

"""

# Filtering out the emails that are Time Sensitive & Deadline-Driven
deadline_emails = yesterday_emails[yesterday_emails['category'] == 'Deadline-Driven Emails']

# @title User Prompt


# Write your user prompt here
user_prompt = f"""

As a email assistant for a given dataframe of Deadline driven emails. Give the summary and next steps.
Today's date is 4 March 2025 and yesterday date is 3 Mar 2025.
Below is the attached DataFrame, which contains all the emails that needs to be summarized:
```
{deadline_emails.to_string()}
```
"""

# @title  Calling the model and display the summary

response_3 = llm(system_prompt, user_prompt)

from IPython.display import display, Markdown
display(Markdown(response_3))

# @title System Prompt
# Write your system prompt here
system_prompt = """

You are an email assistant.
For the recieved email, you are supposed to create the response as per the email's requirement and context.

Keep in mind the below points while writing the email draft.
1. Acknowledge the sender's request.
2. Address the key point mentioned in the original email.
3. Provide next steps, which is required.
4. Write the response in a professional manner.

Fill the details for <> using data provided.

---

Here are the AI-Drafted responsed for Critical Emails recieved yesterday.

Add a line separater here. make below heading bold.

Subject: <Subject of the email>
Sender Name: <Name of the Sender>
AI Drafted Reply: \n<your AI-generated response followed by signature>
---

## rules
1. Strictly response should be as above with heading and bullet heading as bold. 

## Example output:

Here are the AI-Drafted responsed for Critical Emails recieved yesterday.

---

**Subject**: Security Patch Caused System Instability?
**Sender Name**: Mark Davidson
**AI Drafted Reply**:

Mark raises concerns about a drop in detection accuracy following a recent security patch and requests investigation into whether it is a configuration issue or something deeper.

Thanks 
[sender name]


"""

# Filtering out the emails that are Critical Emails, i.e. ('Urgent & High-Priority Emails' + 'Deadline-Driven Emails')
critical_emails = yesterday_emails[yesterday_emails['category'].isin(['Urgent & High-Priority Emails', 'Deadline-Driven Emails'])]

# @title User Prompt

# You may format the classifying method according to you


# Write your user prompt here
user_prompt = f"""

Act as a email assistant draft AI reply for the Alex's emails.
Below is the attached DataFrame, which contains all the critical emails that needs to be replied:
```
{critical_emails.to_string()}
```
"""

# @title  Calling the model and display the summary

response_4 = llm(system_prompt, user_prompt)
from IPython.display import display, Markdown
display(Markdown(response_4))

# @title Evaluation System Prompt

eval_system_prompt = """

You are a LLM acting as a jugde. Your job is to evaluate generated text by another LLM and rate the generated text between 1 - 5.
Where 1 being the lowest and 5 being the highest.
Use the following criteria to evaluate: 

- Relevance: How well does the summary address the input query or task? Identify the key information that is captured or missed. Provide a score from 1 to 5 along with justification.
- Clarity: How clear and understandable is the summary? Highlight any confusing or ambiguous phrases. Provide a score from 1 to 5 along with justification.
- Actionability: Does the summary provide clear next steps or actionable information? Provide a score from 1 to 5 along with justification.

Also, provide insights on
- Strengths: Highlight the key strengths of the summary.
- Improvements: Suggest 1-2 areas for improvement.
- Overall Justification: Provide a 2-3 line summary evaluation, including key observations.

Finally, provide your output in json format.

{
  "Relevance": {"score": "", "justification": ""},
  "Clarity": {"score": "", "justification": ""},
  "Actionability": {"score": "", "justification": ""},
  "Strengths": "",
  "Improvements": "",
  "Overall_Justification": ""
}

"""

# @title Evaluation User Prompt

eval_user_prompt = f"""

evaluate the summary output from the LLM.

"""

# @title Evaluation Function & User Prompt

def evaluate_summary(eval_system_prompt, eval_user_prompt, summary, eval_model="gpt-4o-mini"):
    try:

        modified_prompt = f""" {eval_user_prompt}
                          Here is the summary:
                          ```
                          {summary}
                          ```
                          """

        eval_response = client.chat.completions.create(
            model=eval_model,
            messages=[
                {"role": "system", "content": eval_system_prompt},
                {"role": "user", "content": modified_prompt}
            ],
            temperature=0
        )
        return eval_response.choices[0].message.content.strip()

    except Exception as e:
        print(f"Error evaluating prompt: {e}")
        return "{}"  # Return empty JSON structure on error

response_4

"Here are the AI-Drafted responses for Critical Emails received yesterday.\n\n---\n\n**Subject**: Approval Request: Budget Approval Needed by EOD  \n**Sender Name**: Julia Martin  \n**AI Drafted Reply**:  \n\nThank you for your email, Julia. I acknowledge the urgency of the budget approval needed by the end of today. I will review the attached budget breakdown and provide my approval shortly to ensure we stay on track for the next quarter's projects. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Approval Needed: Project Scope Adjustment for Acme Corp Integration  \n**Sender Name**: Samantha Lee  \n**AI Drafted Reply**:  \n\nThank you for bringing this to my attention, Samantha. I understand the need for a project scope adjustment due to the API limitation with Acme Corp. I approve the proposed change to implement a message queue-based approach. Please proceed with the adjustment and keep me updated on the progress. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: [URGENT] Dashboard Syncing Issues – Production Metrics Missing  \n**Sender Name**: David Whitmore  \n**AI Drafted Reply**:  \n\nHi David,  \n\nI appreciate your prompt notification regarding the dashboard syncing issues. I will have my team investigate the discrepancies in the production metrics immediately. We will aim to provide you with an update within the next 24 hours. If necessary, I will coordinate with our IT team to ensure a swift resolution. \n\nThank you for your patience.  \nBest,  \nAlex  \n\n---\n\n**Subject**: Approval Request: Additional AWS Resources for Project Orion  \n**Sender Name**: Nathan Cole  \n**AI Drafted Reply**:  \n\nHi Nathan,  \n\nThank you for your request regarding additional AWS resources for Project Orion. I understand the importance of scaling up to avoid performance bottlenecks. I will review the proposed solution and provide my approval by March 7 to ensure we maintain our Q2 deadlines. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Blocking Issue Alert – Client Data Sync Failing  \n**Sender Name**: David Kurien  \n**AI Drafted Reply**:  \n\nHi David,  \n\nThank you for alerting me to the client data sync issue. I understand the urgency and will have my team investigate the timeouts and potential code or infrastructure changes immediately. Let's schedule a war room call to discuss the action plan and determine whether a rollback is necessary. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: URGENT: Approval Needed for 2-Week Extension on Acme Corp Deployment  \n**Sender Name**: Tanya Patel  \n**AI Drafted Reply**:  \n\nHi Tanya,  \n\nThank you for your email regarding the extension for the Acme Corp deployment. I understand the reasons for the delay and agree that it is essential to avoid rushed delivery. I approve the two-week extension to ensure quality testing. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: System Crashing During Shift Changes – URGENT  \n**Sender Name**: David Whitmore  \n**AI Drafted Reply**:  \n\nHi David,  \n\nI appreciate your urgent notification about the system crashing during shift changes. I will have my team investigate the issue immediately. I look forward to our call at 3 PM EST to discuss this further and ensure we resolve it as quickly as possible. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Security Risk – Critical Patch Delayed  \n**Sender Name**: Bob Smith  \n**AI Drafted Reply**:  \n\nHi Bob,  \n\nThank you for bringing the security patch delay to my attention. I understand the risks involved and the urgency of the situation. Please proceed with the manual workaround while we assess the situation further. I will discuss with the team to determine the best course of action regarding the patch. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: URGENT: Production Halt – Machine Control System Unresponsive  \n**Sender Name**: David Whitmore  \n**AI Drafted Reply**:  \n\nHi David,  \n\nThank you for your urgent message regarding the machine control system. I understand the critical nature of this issue and will have my team address it immediately. I will call you shortly to discuss the situation and ensure we restore operations as soon as possible. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: [High Priority] Authentication Failing for Multiple Users  \n**Sender Name**: Mark Davidson  \n**AI Drafted Reply**:  \n\nHi Mark,  \n\nThank you for informing me about the authentication issue affecting multiple users. I will investigate whether this is a known issue or a misconfiguration. Rest assured, we will work to restore access as quickly as possible. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Approval Needed: Purchase of Design Tool for Engineering Team  \n**Sender Name**: Liam Ross  \n**AI Drafted Reply**:  \n\nHi Liam,  \n\nThank you for your request regarding the purchase of Adobe Creative Cloud licenses. I will review the justification and get back to you shortly regarding the approval. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: URGENT: Approval for Security Audit Vendor – Time-Sensitive  \n**Sender Name**: Rachel Lim  \n**AI Drafted Reply**:  \n\nHi Rachel,  \n\nThank you for the reminder regarding the security audit contract with CyberShield. I understand the urgency and will provide my approval by March 7 to ensure compliance certification renewal. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: URGENT: Critical System Downtime – Immediate Attention Required  \n**Sender Name**: David Whitmore  \n**AI Drafted Reply**:  \n\nHi David,  \n\nThank you for alerting me to the major outage with the Orion Analytics Dashboard. I will have my team investigate this issue immediately and provide a status update by 12 PM EST today. If necessary, I am available for a call to discuss further. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Follow-Up: Server Downtime - Critical Fix Required  \n**Sender Name**: Bob Smith  \n**AI Drafted Reply**:  \n\nHi Bob,  \n\nThank you for your follow-up regarding the server downtime. I will review the attached logs and provide any insights that could expedite the fix. I appreciate your team's efforts in resolving this issue. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Approval Request: Dev Environment Upgrade for Faster Builds  \n**Sender Name**: Kevin Tran  \n**AI Drafted Reply**:  \n\nHi Kevin,  \n\nThank you for your request regarding the upgrade of the development environment. I understand the need for faster builds and will review the proposed upgrade to provide my approval by March 10. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: URGENT: Overdue Invoice Approval for Cloud Services  \n**Sender Name**: Sarah Mitchell  \n**AI Drafted Reply**:  \n\nHi Sarah,  \n\nThank you for bringing the overdue invoice to my attention. I will prioritize the approval to avoid any late fees or service disruptions. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Firewall Logs Disappeared – What’s Going On?  \n**Sender Name**: Mark Davidson  \n**AI Drafted Reply**:  \n\nHi Mark,  \n\nThank you for notifying me about the missing firewall logs. I will investigate whether this is a storage issue or data corruption and will keep you updated on the findings. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: URGENT: Medication Alerts Not Firing – This is Dangerous  \n**Sender Name**: Rachel Thompson  \n**AI Drafted Reply**:  \n\nHi Rachel,  \n\nThank you for your urgent message regarding the medication alerts. I understand the seriousness of this issue and will address it immediately. I look forward to our emergency call in 30 minutes to discuss the next steps. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Approval Request: Travel Budget for AWS Summit 2025  \n**Sender Name**: Olivia Chen  \n**AI Drafted Reply**:  \n\nHi Olivia,  \n\nThank you for your request regarding the travel budget for the AWS Summit 2025. I will review the estimated costs and provide my approval shortly. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Pending Approval – Invoice Dispute (Microsoft Teams)  \n**Sender Name**: Alice Johnson  \n**AI Drafted Reply**:  \n\nHi Alice,  \n\nThank you for the reminder about the invoice approval discussion. I will join the Microsoft Teams meeting at the scheduled time to address the unresolved issues. \n\nBest,  \nAlex  \n\n---\n\n**Subject**: Urgent: Performance Degradation in Production System  \n**Sender Name**: Nathan Ellis  \n**AI Drafted Reply**:  \n\nHi Nathan,  \n\nThank you for bringing the performance degradation to my attention. I approve the emergency scale-up of the database instance. Please proceed with client communication as you see fit, and let’s discuss further in the Google Meet scheduled for 2:30 PM EST. \n\nBest,  \nAlex  \n\n---"

responses = response_4.split("---")            # splitting response on the basis of delimiter, so we get each response for the email indivisually
for _ in responses[1:-1]:                      # Excluding First and Last Element as they are nothing but blanks .i.e. (' ')
  display(Markdown(_))
  print("+"*100)

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

# @title Evaluation for Each Generated Response
evaluation_results = [evaluate_summary(eval_system_prompt, eval_user_prompt, summary) for summary in responses[1:-1]]        # Excluding first and last elements as they are not the actual emails and some LLM generated support texts

display(Markdown(evaluation_results[0]))

# @title Converting JSON Scores into DataFrame
import pandas as pd
import json

scores = []
justifications = []
strengths = []
improvements = []


for result in evaluation_results:
    try:
        result_dict = json.loads(result)
        score_dict = {k: v.get("score", "NA") for k, v in result_dict.items() if isinstance(v, dict) and "score" in v}
        justification = result_dict.get("Overall_Justification", "NA")
        strength = result_dict.get("Strengths", "NA")
        improvement = result_dict.get("Improvements", "NA")

        # Append Results
        scores.append(score_dict)
        justifications.append(justification)
        strengths.append(strength)
        improvements.append(improvement)

    except (json.JSONDecodeError, KeyError, TypeError):
        scores.append({"Relevance": None, "Clarity": None, "Conciseness": None, "Coherence": None, "Actionability": None})
        justifications.append("")
        strengths.append("")
        improvements.append("")

# @title Creating Final Evaluation DataFrame
pd.set_option('max_colwidth', 0)
df_scores = pd.DataFrame(scores)
df_scores["Strengths"] = strengths
df_scores["Improvements"] = improvements
df_scores["Justification"] = justifications

# @title Display Final Evaluation Table
df_scores

	Relevance	Clarity	Actionability	Strengths	Improvements	Justification
0	5	5	4	The summary effectively captures the urgency of the request and provides a clear response from the recipient, maintaining a professional tone throughout.	It could include a specific timeframe for when Julia can expect the approval or any additional information needed from her side to enhance clarity on next steps.	The summary is highly relevant and clear, effectively communicating the urgency of the budget approval. However, it could benefit from more explicit next steps to enhance actionability.
1	5	5	5	The summary effectively captures the essence of the communication, providing a clear approval and rationale while outlining next steps.	While the summary is strong, it could include a brief mention of the timeline for the adjustment or any specific metrics for progress updates to enhance clarity on expectations.	The summary is highly relevant, clear, and actionable, effectively conveying the necessary information for the project scope adjustment. It meets all evaluation criteria well.
2	5	5	4	The summary effectively captures the urgency of the situation and provides a clear response from Alex, including a commitment to follow up within a specific timeframe.	It could include a brief mention of any information or assistance needed from David to facilitate the investigation, enhancing the actionable aspect.	The summary is highly relevant and clear, effectively addressing the issue at hand. While it provides actionable information, it could be improved by suggesting any potential actions for the sender.
3	5	5	5	The summary effectively captures the essence of the communication, maintains a professional tone, and provides a clear timeline for action.	While the summary is strong, it could include a brief mention of what specific AWS resources are being requested for added context. Additionally, it could specify any potential impacts of not approving the request in a timely manner.	The summary is highly relevant, clear, and actionable, effectively communicating the necessary information regarding the approval request. It successfully outlines the next steps while maintaining a professional tone.
4	5	5	5	The summary effectively captures the urgency of the situation, provides a clear response, and outlines actionable next steps.	While the summary is strong, it could include a specific time frame for the investigation or the proposed call to enhance urgency and clarity further.	The summary is highly relevant, clear, and actionable, effectively addressing the client data sync issue while providing a structured response.
5	5	5	5	The summary effectively captures the key elements of the communication, including the subject, sender, and the approval message, while maintaining clarity and directness.	While the summary is strong, it could include a brief mention of any specific actions that need to be taken next or a deadline for the extension to enhance clarity further.	The summary is highly relevant, clear, and actionable, effectively conveying the necessary information regarding the extension approval. It meets all evaluation criteria well.
6	5	5	4	The summary effectively conveys urgency and a commitment to resolving the issue. It also establishes a clear line of communication with a scheduled call.	The summary could include specific actions that will be taken during the investigation to enhance clarity on next steps. Additionally, it could mention any immediate measures to mitigate the impact of the crashes until a solution is found.	The summary is highly relevant and clear, effectively addressing the urgent issue raised. While it provides a good foundation for action, it could benefit from more detailed next steps.
7	4	5	4	The summary effectively communicates the urgency of the situation and provides a clear response to the sender. It maintains a professional tone and structure.	1. Include more specific details about the risks associated with the delayed patch. 2. Provide a timeline or specific actions for the assessment process to enhance clarity on next steps.	The summary is relevant and clear, effectively addressing the issue at hand. While it provides actionable steps, it could benefit from additional details to enhance its relevance and actionability.
8	5	5	4	The summary effectively conveys urgency and a commitment to resolving the issue, maintaining a professional tone throughout.	It could include specific next steps or a timeline for resolution to enhance actionability. Additionally, mentioning any immediate measures being taken could provide more context.	The summary is highly relevant and clear, effectively communicating the urgency of the situation. While it provides a good response, adding more specific actionable steps would improve its effectiveness.
9	5	5	4	The summary effectively captures the essence of the communication, maintains a professional tone, and reassures the sender that the issue will be addressed promptly.	The summary could include a specific timeline for when the sender can expect an update or resolution, and it could mention any immediate steps users should take while the issue is being investigated.	The summary is highly relevant and clear, effectively communicating the situation and next steps. However, adding more specific actionable details could enhance its effectiveness.
10	4	5	4	The summary effectively captures the essence of the communication, maintaining a professional tone and clear structure.	It could include a specific timeframe for when the sender will respond, and it might be beneficial to mention any additional information needed for the approval process.	The summary is relevant, clear, and mostly actionable, effectively conveying the necessary information while maintaining professionalism. However, adding more specificity regarding the timeline for approval would enhance its effectiveness.
11	5	5	5	The summary effectively captures the urgency of the situation, includes relevant details about the vendor, and presents a clear response from the sender.	While the summary is strong, it could include a brief mention of the consequences of not approving the contract on time to emphasize the urgency further. Additionally, a subject line could be more descriptive to reflect the content of the email better.	The summary is highly relevant, clear, and actionable, effectively conveying the necessary information regarding the approval for the security audit vendor. It meets all evaluation criteria well.
12	5	5	5	The summary effectively captures the urgency of the situation, provides a clear response, and outlines next steps, making it highly relevant and actionable.	While the summary is strong, it could include a brief mention of the potential impact of the downtime on users or operations to provide more context. Additionally, specifying the time zone for the status update could enhance clarity for recipients in different regions.	The summary is highly effective, addressing the critical issue with clarity and providing actionable next steps. It successfully communicates the urgency and response required, making it a strong output.
13	4	5	4	The summary is well-structured and maintains a professional tone. It effectively acknowledges the follow-up and expresses appreciation for the efforts made to resolve the issue.	1. Include more specific details about the server downtime to enhance relevance. 2. Suggest a timeline for when the insights will be provided to improve actionability.	The summary is relevant and clear, with actionable next steps, but could benefit from additional details and a sense of urgency regarding the server downtime.
14	4	5	4	The summary is well-structured and clearly conveys the essential information regarding the approval request and response. It maintains a professional tone appropriate for business communication.	1. Include more details about the specific upgrades being proposed to enhance relevance. 2. Suggest any actions Kevin should take while waiting for the approval to improve actionability.	The summary is strong in clarity and structure, effectively communicating the essential elements of the approval request. However, it could benefit from additional details and actionable steps to enhance its relevance and actionability.
15	4	5	4	The summary is concise and maintains a professional tone. It clearly acknowledges the issue and indicates a commitment to resolving it promptly.	1. Include more specific details about the invoice, such as the amount or services involved. 2. Provide a timeline for when the approval will be completed to enhance actionability.	The summary is relevant, clear, and actionable, but it could benefit from additional details and a specific timeline to improve its effectiveness.
16	4	5	4	The summary is concise and directly addresses the issue at hand. It maintains a professional tone and provides a clear response to the inquiry.	The summary could benefit from including more specific details about the investigation process or potential next steps for the recipient. Additionally, mentioning a timeline for updates would enhance clarity and actionability.	The summary effectively communicates the main points regarding the missing firewall logs and the intended investigation. While it is clear and relevant, it could be improved by providing more actionable details and context.
17	4	5	4	The summary effectively captures the urgency of the situation and presents a clear and polite response. The structure is well-organized, making it easy to follow.	The summary could benefit from including more specific details about the medication alerts and outlining any immediate actions that will be taken prior to the call.	Overall, the summary is relevant, clear, and actionable, but it could be improved by providing more specific details and immediate actions related to the medication alerts.
18	4	5	3	The summary is well-structured and clearly presents the essential information regarding the approval request and response.	1. Include more details about the travel budget or context of the AWS Summit to enhance relevance. 2. Provide a timeline or specific next steps for the approval process to improve actionability.	The summary is relevant and clear, effectively conveying the essential information. However, it could benefit from additional details and clearer next steps to enhance its overall effectiveness.
19	4	5	4	The summary is concise and well-structured, providing essential information in a clear manner. It effectively communicates the intent to resolve the invoice dispute.	1. Include more details about the specific unresolved issues to enhance relevance. 2. Suggest specific next steps or questions that Alex might consider addressing in the meeting for better actionability.	The summary is relevant and clear, effectively conveying the main points of the email. However, it could benefit from additional context and specific next steps to enhance its overall effectiveness.
20	5	5	5	The summary effectively captures the urgency of the situation, provides a clear response, and outlines next steps, making it highly relevant and actionable.	While the summary is strong, it could include a brief mention of the specific actions Nathan should take in client communication for added clarity. Additionally, specifying the platform for the Google Meet could enhance clarity.	The summary is highly effective, addressing the key points of relevance, clarity, and actionability. It successfully communicates the necessary information while maintaining a professional tone.

Business Problem - Automating Email Prioritization & Responses¶

The Case Study¶

Objective¶

LLM Setup¶

Data Setup¶

Background¶

About You¶

TASK - Categorization of emails¶

Task 1: Awareness of your email: Get to know the detailed summary of your email received¶

1A. Executive Dashboard (Top-Level Summary of Yesterday’s Inbox)¶

TASK¶

1B. Urgent Emails from Yesterday (🛑 Must-Do First Today)¶

TASK¶

1C: Deadline-Driven Emails from Yesterday (⚡ Needs Attention Today)¶

TASK¶

Task 2: AI-Generated "First Response" Drafts for Critical Email¶

Critical Emails are combination of the Urgent & High-Priority Emails + Deadline-Driven Emails¶

TASK¶

Task 3: Evaluation¶

Evaluation of Prompt Outputs with LLM-as-a-Judge¶

2. Evaluation Criteria¶

TASK¶

Task 4: Summary and Recommendation¶

Summary and Recommendation¶

	date_received	sender	subject	body	main_recipient
email_id
1	3/3/2025	Julia Martin	Approval Request: Budget Approval Needed by EOD	Hi Alex,\r\n\r\nI hope you're doing well. As w...	Alex
2	3/3/2025	Fiona White	Are Your APIs Secure? Reddit & Discord Sound t...	Hi Alex,\r\n\r\nA heated Discord discussion in...	Alex
3	3/3/2025	Samantha Lee	Approval Needed: Project Scope Adjustment for ...	Hi Alex,\r\n\r\nWeve encountered an unexpecte...	Alex
4	3/3/2025	James Patel	Subject: Daily Update Project Titan (March 3)	Hey Alex,\r\n\r\nQuick update on Project Titan...	Alex
5	3/3/2025	David Whitmore	[URGENT] Dashboard Syncing Issues Production...	Hey Alex,\r\n\r\nWeve got a big issue right n...	Alex